# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import re

import scrapy
from scrapy.loader import ItemLoader
from itemloaders.processors import TakeFirst, MapCompose, Join, Identity


class CnblogsItemLoader(ItemLoader):
    default_output_processor = TakeFirst()


# 日期格式正好，如果需要调整就用这个函数
def date_convert(value):
    # match_date = re.compile(r'\d{4}-\d{2}-\d{2}')
    match_date = re.match(r'.*?(\d+.*)', value)
    if match_date:
        return match_date.group(1)
    return "1111-1-1"


class CnblogsNewsItem(scrapy.Item):
    url = scrapy.Field()
    id = scrapy.Field()
    title = scrapy.Field()
    create_time = scrapy.Field(
        input_processor=MapCompose(date_convert),
    )
    content = scrapy.Field()
    tags = scrapy.Field(
        output_processor=Join(","),
    )
    front_image_url = scrapy.Field(
        output_processor=Identity()
    )
    front_image_path = scrapy.Field()
    commentCount = scrapy.Field()
    diggCount = scrapy.Field()
    totalView = scrapy.Field()
    md5_id = scrapy.Field()

    def get_insert_sql(self):
        insert_sql = """
            insert into `cn_news` (`url`, `id`, `title`, `create_time`, 
            `content`, `tags`, `front_image_url`, `front_image_path`, 
            `commentCount`, `diggCount`, `totalView`, `md5_id`)
            values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            on duplicate key update `title`=values(title), `content`=values(content), commentCount=values(commentCount),
            diggCount=values(diggCount), `totalView`=values(totalView)
        """  # 最后两行是当主键冲突的时候强制更新
        params = (
            self.get('url', ""),
            self.get('id', 0),
            self.get('title', ""),
            self.get('create_time', "1111-1-1"),
            self.get('content', ""),
            self.get('tags', ""),
            self.get(",".join(self.get('front_image_url', []))),
            self.get('front_image_path', ""),
            self.get('commentCount', 0),
            self.get('diggCount', 0),
            self.get('totalView', 0),
            self.get('md5_id', ""),
        )
        return insert_sql, params
